import re, urllib, urllib2

def LoadUrlRef(url,refurl):
       url=url.strip()
       url=url.replace(' ','%20')
       print 'LoadUrl: '+url

       user_agent = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'
       req = urllib2.Request(url)
       req.add_header('User-Agent', user_agent)
       req.add_header('Referer', refurl)

       response = urllib2.urlopen(req)
       html = response.read()
       response.close()
       if len(html)>0:
          html = html.strip()
       return html

def LoadUrl(url):
       url=url.strip()
       url=url.replace(' ','%20')
       print 'LoadUrl: '+url

       user_agent = 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'
       req = urllib2.Request(url)
       req.add_header('User-Agent', user_agent)

       response = urllib2.urlopen(req)
       html = response.read()
       response.close()
       if len(html)>0:
          html = html.strip()
       return html

def GetAllValues(string,key):
        prog = re.compile('<'+key+'>(.+?)<\/'+key+'>',flags=re.DOTALL|re.I)
        result = prog.findall(string)
        return result
    
def GetValue(string,key):
        allValues = GetAllValues(string,key)
        result = ''
        try:
                result = allValues[0]
        except:
                result = ''
        return result

def GetParams(params):
    if len(params)>1:
        cleanedparams=params
        if (params[len(params)-1]=='/'):
                params=params[0:len(params)-2]
        pairsofparams=cleanedparams.split('|')
        param={}
        for i in range(len(pairsofparams)):
                for key, value in re.findall('(.*?)=(.*)',pairsofparams[i]):
                    param[key]=value
                                
        return param

def DictToUrl(params):
        result = ''
        for key in params:
               result=result+'|'+key+'='+params[key]
        return result

def RemoveUnicodeCodes(s):
    match = re.compile('&#(.*?);').findall(s)
    for unic in match:
        if int(unic)<=256:
            s = s.replace('&#'+unic+';',chr(int(unic)))
    return s
